{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/15, 11:24:58, Iteration: 100, Train Loss: 5.2976e+00, c1: 2.3155e-01, c2: 1.6773e-03, d1: 5.0493e-01, d2: 0.0000e+00, s: 4.8072e-02, l1_reg: 3.2540e+02, l2_reg: 5.1365e+01\n",
      "2020/09/15, 11:24:59, Iteration: 200, Train Loss: 2.2072e+00, c1: 5.4056e-01, c2: 2.3267e-03, d1: 1.6434e-01, d2: 0.0000e+00, s: 2.1244e-02, l1_reg: 3.2962e+02, l2_reg: 5.3059e+01\n",
      "2020/09/15, 11:25:00, Iteration: 300, Train Loss: 2.0222e+00, c1: 3.3661e-01, c2: 4.4059e-03, d1: 1.6415e-01, d2: 0.0000e+00, s: 8.4310e-03, l1_reg: 3.2918e+02, l2_reg: 5.2826e+01\n",
      "2020/09/15, 11:25:01, Iteration: 400, Train Loss: 1.9341e+00, c1: 2.8967e-01, c2: 6.2358e-04, d1: 1.6382e-01, d2: 0.0000e+00, s: 6.9505e-03, l1_reg: 3.2894e+02, l2_reg: 5.2777e+01\n",
      "2020/09/15, 11:25:02, Iteration: 500, Train Loss: 1.9640e+00, c1: 2.9080e-01, c2: 3.9281e-03, d1: 1.6339e-01, d2: 0.0000e+00, s: 5.9406e-03, l1_reg: 3.2885e+02, l2_reg: 5.2781e+01\n",
      "2020/09/15, 11:25:02, Iteration: 600, Train Loss: 1.8211e+00, c1: 1.6843e-01, c2: 2.3830e-03, d1: 1.6288e-01, d2: 0.0000e+00, s: 6.3860e-03, l1_reg: 3.2878e+02, l2_reg: 5.2822e+01\n",
      "2020/09/15, 11:25:03, Iteration: 700, Train Loss: 1.8992e+00, c1: 2.3032e-01, c2: 4.6416e-03, d1: 1.6224e-01, d2: 0.0000e+00, s: 6.1091e-03, l1_reg: 3.2876e+02, l2_reg: 5.2922e+01\n",
      "2020/09/15, 11:25:04, Iteration: 800, Train Loss: 1.8703e+00, c1: 2.1077e-01, c2: 4.5406e-03, d1: 1.6141e-01, d2: 0.0000e+00, s: 6.6773e-03, l1_reg: 3.2882e+02, l2_reg: 5.3084e+01\n",
      "2020/09/15, 11:25:05, Iteration: 900, Train Loss: 1.8844e+00, c1: 1.7088e-01, c2: 1.1055e-02, d1: 1.6030e-01, d2: 0.0000e+00, s: 7.4781e-03, l1_reg: 3.2892e+02, l2_reg: 5.3307e+01\n",
      "2020/09/15, 11:25:06, Iteration: 1000, Train Loss: 1.7765e+00, c1: 1.4961e-01, c2: 4.0269e-03, d1: 1.5866e-01, d2: 0.0000e+00, s: 8.3572e-03, l1_reg: 3.2911e+02, l2_reg: 5.3592e+01\n",
      "2020/09/15, 11:25:06, Iteration: 1100, Train Loss: 1.7526e+00, c1: 1.6869e-01, c2: 2.3147e-03, d1: 1.5608e-01, d2: 0.0000e+00, s: 9.7483e-03, l1_reg: 3.2940e+02, l2_reg: 5.3992e+01\n",
      "2020/09/15, 11:25:07, Iteration: 1200, Train Loss: 1.5810e+00, c1: 5.2440e-02, c2: 1.5120e-03, d1: 1.5135e-01, d2: 0.0000e+00, s: 9.6234e-03, l1_reg: 3.2999e+02, l2_reg: 5.4606e+01\n",
      "2020/09/15, 11:25:08, Iteration: 1300, Train Loss: 1.6034e+00, c1: 2.0580e-01, c2: 1.9778e-03, d1: 1.3779e-01, d2: 0.0000e+00, s: 1.1277e-02, l1_reg: 3.3145e+02, l2_reg: 5.5824e+01\n",
      "2020/09/15, 11:25:09, Iteration: 1400, Train Loss: 9.0416e-01, c1: 9.3514e-02, c2: 1.5259e-03, d1: 7.9539e-02, d2: 0.0000e+00, s: 1.0509e-02, l1_reg: 3.3508e+02, l2_reg: 5.8918e+01\n",
      "2020/09/15, 11:25:10, Iteration: 1500, Train Loss: 5.3297e-01, c1: 6.1112e-02, c2: 2.8357e-03, d1: 4.4350e-02, d2: 0.0000e+00, s: 1.0038e-02, l1_reg: 3.3686e+02, l2_reg: 6.0853e+01\n",
      "2020/09/15, 11:25:10, Iteration: 1600, Train Loss: 3.3820e-01, c1: 4.3505e-02, c2: 1.4584e-03, d1: 2.8011e-02, d2: 0.0000e+00, s: 9.5265e-03, l1_reg: 3.3750e+02, l2_reg: 6.2237e+01\n",
      "2020/09/15, 11:25:11, Iteration: 1700, Train Loss: 3.1240e-01, c1: 3.2539e-02, c2: 4.3807e-03, d1: 2.3605e-02, d2: 0.0000e+00, s: 9.0533e-03, l1_reg: 3.3786e+02, l2_reg: 6.3096e+01\n",
      "2020/09/15, 11:25:12, Iteration: 1800, Train Loss: 3.1639e-01, c1: 9.5319e-02, c2: 4.7680e-04, d1: 2.1630e-02, d2: 0.0000e+00, s: 9.3845e-03, l1_reg: 3.3781e+02, l2_reg: 6.3544e+01\n",
      "2020/09/15, 11:25:13, Iteration: 1900, Train Loss: 3.0343e-01, c1: 4.9683e-02, c2: 4.5900e-03, d1: 2.0784e-02, d2: 0.0000e+00, s: 9.4858e-03, l1_reg: 3.3777e+02, l2_reg: 6.3970e+01\n",
      "2020/09/15, 11:25:14, Iteration: 2000, Train Loss: 2.7418e-01, c1: 6.5873e-02, c2: 2.6013e-03, d1: 1.8229e-02, d2: 0.0000e+00, s: 1.1092e-02, l1_reg: 3.3777e+02, l2_reg: 6.4399e+01\n",
      "2020/09/15, 11:25:15, Iteration: 2100, Train Loss: 2.5674e-01, c1: 8.6691e-02, c2: 5.6266e-04, d1: 1.6442e-02, d2: 0.0000e+00, s: 1.0896e-02, l1_reg: 3.3771e+02, l2_reg: 6.4750e+01\n",
      "2020/09/15, 11:25:15, Iteration: 2200, Train Loss: 2.3135e-01, c1: 5.1745e-02, c2: 2.1865e-03, d1: 1.5774e-02, d2: 0.0000e+00, s: 8.9633e-03, l1_reg: 3.3776e+02, l2_reg: 6.5142e+01\n",
      "2020/09/15, 11:25:16, Iteration: 2300, Train Loss: 2.3671e-01, c1: 5.9999e-02, c2: 3.0838e-03, d1: 1.4587e-02, d2: 0.0000e+00, s: 9.2862e-03, l1_reg: 3.3773e+02, l2_reg: 6.5410e+01\n",
      "2020/09/15, 11:25:17, Iteration: 2400, Train Loss: 2.0976e-01, c1: 4.2438e-02, c2: 2.2385e-03, d1: 1.4493e-02, d2: 0.0000e+00, s: 8.5918e-03, l1_reg: 3.3770e+02, l2_reg: 6.5707e+01\n",
      "2020/09/15, 11:25:18, Iteration: 2500, Train Loss: 2.4952e-01, c1: 6.8179e-02, c2: 2.4496e-03, d1: 1.5684e-02, d2: 0.0000e+00, s: 8.6378e-03, l1_reg: 3.3766e+02, l2_reg: 6.5926e+01\n",
      "2020/09/15, 11:25:19, Iteration: 2600, Train Loss: 2.0075e-01, c1: 4.0059e-02, c2: 1.3107e-03, d1: 1.4758e-02, d2: 0.0000e+00, s: 8.6998e-03, l1_reg: 3.3764e+02, l2_reg: 6.6134e+01\n",
      "2020/09/15, 11:25:19, Iteration: 2700, Train Loss: 1.8001e-01, c1: 3.9259e-02, c2: 6.1244e-04, d1: 1.3462e-02, d2: 0.0000e+00, s: 8.0679e-03, l1_reg: 3.3752e+02, l2_reg: 6.6207e+01\n",
      "2020/09/15, 11:25:20, Iteration: 2800, Train Loss: 1.9836e-01, c1: 4.6815e-02, c2: 1.6571e-03, d1: 1.3497e-02, d2: 0.0000e+00, s: 8.2126e-03, l1_reg: 3.3744e+02, l2_reg: 6.6328e+01\n",
      "2020/09/15, 11:25:21, Iteration: 2900, Train Loss: 2.0640e-01, c1: 5.7581e-02, c2: 6.1631e-04, d1: 1.4265e-02, d2: 0.0000e+00, s: 8.1877e-03, l1_reg: 3.3729e+02, l2_reg: 6.6419e+01\n",
      "2020/09/15, 11:25:22, Iteration: 3000, Train Loss: 2.1645e-01, c1: 5.1169e-02, c2: 3.3590e-03, d1: 1.3169e-02, d2: 0.0000e+00, s: 8.8679e-03, l1_reg: 3.3719e+02, l2_reg: 6.6513e+01\n",
      "2020/09/15, 11:25:23, Iteration: 3100, Train Loss: 2.8056e-01, c1: 9.3807e-02, c2: 4.1046e-03, d1: 1.4570e-02, d2: 0.0000e+00, s: 8.1737e-03, l1_reg: 3.3708e+02, l2_reg: 6.6589e+01\n",
      "2020/09/15, 11:25:24, Iteration: 3200, Train Loss: 2.1323e-01, c1: 4.2163e-02, c2: 4.0435e-03, d1: 1.3063e-02, d2: 0.0000e+00, s: 9.6155e-03, l1_reg: 3.3696e+02, l2_reg: 6.6656e+01\n",
      "2020/09/15, 11:25:24, Iteration: 3300, Train Loss: 1.8057e-01, c1: 1.8365e-02, c2: 2.7775e-03, d1: 1.3442e-02, d2: 0.0000e+00, s: 9.7487e-03, l1_reg: 3.3685e+02, l2_reg: 6.6731e+01\n",
      "2020/09/15, 11:25:25, Iteration: 3400, Train Loss: 2.3304e-01, c1: 7.1215e-02, c2: 3.1693e-03, d1: 1.3013e-02, d2: 0.0000e+00, s: 8.2176e-03, l1_reg: 3.3674e+02, l2_reg: 6.6793e+01\n",
      "2020/09/15, 11:25:26, Iteration: 3500, Train Loss: 2.0591e-01, c1: 2.6221e-02, c2: 4.9452e-03, d1: 1.3023e-02, d2: 0.0000e+00, s: 9.5799e-03, l1_reg: 3.3664e+02, l2_reg: 6.6870e+01\n",
      "2020/09/15, 11:25:27, Iteration: 3600, Train Loss: 2.0317e-01, c1: 4.2897e-02, c2: 2.1057e-03, d1: 1.3921e-02, d2: 0.0000e+00, s: 8.1656e-03, l1_reg: 3.3652e+02, l2_reg: 6.6916e+01\n",
      "2020/09/15, 11:25:28, Iteration: 3700, Train Loss: 2.2205e-01, c1: 6.1472e-02, c2: 1.0333e-03, d1: 1.5024e-02, d2: 0.0000e+00, s: 7.9529e-03, l1_reg: 3.3638e+02, l2_reg: 6.6907e+01\n",
      "2020/09/15, 11:25:29, Iteration: 3800, Train Loss: 1.8569e-01, c1: 3.7938e-02, c2: 6.7340e-04, d1: 1.4101e-02, d2: 0.0000e+00, s: 7.5011e-03, l1_reg: 3.3622e+02, l2_reg: 6.6935e+01\n",
      "2020/09/15, 11:25:29, Iteration: 3900, Train Loss: 2.2921e-01, c1: 5.6211e-02, c2: 2.2969e-03, d1: 1.5003e-02, d2: 0.0000e+00, s: 7.0155e-03, l1_reg: 3.3610e+02, l2_reg: 6.6968e+01\n",
      "2020/09/15, 11:25:30, Iteration: 4000, Train Loss: 2.1801e-01, c1: 7.2125e-02, c2: 7.6935e-04, d1: 1.3818e-02, d2: 0.0000e+00, s: 6.2252e-03, l1_reg: 3.3600e+02, l2_reg: 6.6970e+01\n",
      "2020/09/15, 11:25:31, Iteration: 4100, Train Loss: 1.7817e-01, c1: 4.1099e-02, c2: 7.3494e-04, d1: 1.2972e-02, d2: 0.0000e+00, s: 6.6849e-03, l1_reg: 3.3587e+02, l2_reg: 6.6956e+01\n",
      "2020/09/15, 11:25:32, Iteration: 4200, Train Loss: 1.9455e-01, c1: 3.3826e-02, c2: 1.3349e-03, d1: 1.4738e-02, d2: 0.0000e+00, s: 6.8962e-03, l1_reg: 3.3576e+02, l2_reg: 6.6957e+01\n",
      "2020/09/15, 11:25:33, Iteration: 4300, Train Loss: 1.6550e-01, c1: 2.6866e-02, c2: 7.8445e-04, d1: 1.3079e-02, d2: 0.0000e+00, s: 6.5393e-03, l1_reg: 3.3562e+02, l2_reg: 6.6936e+01\n",
      "2020/09/15, 11:25:33, Iteration: 4400, Train Loss: 1.6326e-01, c1: 1.4273e-02, c2: 1.2517e-03, d1: 1.3646e-02, d2: 0.0000e+00, s: 5.7944e-03, l1_reg: 3.3548e+02, l2_reg: 6.6956e+01\n",
      "2020/09/15, 11:25:34, Iteration: 4500, Train Loss: 1.9985e-01, c1: 2.6564e-02, c2: 3.8317e-03, d1: 1.3497e-02, d2: 0.0000e+00, s: 6.0921e-03, l1_reg: 3.3532e+02, l2_reg: 6.6955e+01\n",
      "2020/09/15, 11:25:35, Iteration: 4600, Train Loss: 1.9604e-01, c1: 3.7635e-02, c2: 1.9908e-03, d1: 1.3849e-02, d2: 0.0000e+00, s: 5.7825e-03, l1_reg: 3.3522e+02, l2_reg: 6.6962e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/15, 11:25:36, Iteration: 4700, Train Loss: 1.9377e-01, c1: 3.6629e-02, c2: 2.8166e-03, d1: 1.2897e-02, d2: 0.0000e+00, s: 6.7103e-03, l1_reg: 3.3505e+02, l2_reg: 6.6934e+01\n",
      "2020/09/15, 11:25:37, Iteration: 4800, Train Loss: 2.4371e-01, c1: 8.5448e-02, c2: 2.0999e-03, d1: 1.3726e-02, d2: 0.0000e+00, s: 6.7094e-03, l1_reg: 3.3488e+02, l2_reg: 6.6908e+01\n",
      "2020/09/15, 11:25:37, Iteration: 4900, Train Loss: 2.0002e-01, c1: 4.1272e-02, c2: 1.9829e-03, d1: 1.3892e-02, d2: 0.0000e+00, s: 6.0308e-03, l1_reg: 3.3480e+02, l2_reg: 6.6933e+01\n",
      "2020/09/15, 11:25:38, Iteration: 5000, Train Loss: 2.0104e-01, c1: 4.1047e-02, c2: 2.2355e-03, d1: 1.3763e-02, d2: 0.0000e+00, s: 5.3966e-03, l1_reg: 3.3463e+02, l2_reg: 6.6924e+01\n",
      "2020/09/15, 11:25:39, Iteration: 5100, Train Loss: 1.8783e-01, c1: 3.6393e-02, c2: 1.9223e-03, d1: 1.3221e-02, d2: 0.0000e+00, s: 5.1181e-03, l1_reg: 3.3450e+02, l2_reg: 6.6888e+01\n",
      "2020/09/15, 11:25:40, Iteration: 5200, Train Loss: 2.0733e-01, c1: 3.9892e-02, c2: 2.8016e-03, d1: 1.3942e-02, d2: 0.0000e+00, s: 5.6378e-03, l1_reg: 3.3435e+02, l2_reg: 6.6839e+01\n",
      "2020/09/15, 11:25:41, Iteration: 5300, Train Loss: 1.8116e-01, c1: 9.5768e-03, c2: 5.4183e-04, d1: 1.6616e-02, d2: 0.0000e+00, s: 3.9178e-03, l1_reg: 3.3420e+02, l2_reg: 6.6843e+01\n",
      "2020/09/15, 11:25:41, Iteration: 5400, Train Loss: 1.6234e-01, c1: 1.3888e-02, c2: 1.3257e-03, d1: 1.3519e-02, d2: 0.0000e+00, s: 4.6616e-03, l1_reg: 3.3407e+02, l2_reg: 6.6846e+01\n",
      "2020/09/15, 11:25:42, Iteration: 5500, Train Loss: 1.4397e-01, c1: 1.1402e-02, c2: 5.7210e-04, d1: 1.2684e-02, d2: 0.0000e+00, s: 4.6390e-03, l1_reg: 3.3399e+02, l2_reg: 6.6854e+01\n",
      "2020/09/15, 11:25:43, Iteration: 5600, Train Loss: 1.7768e-01, c1: 1.6764e-02, c2: 1.0646e-03, d1: 1.5026e-02, d2: 0.0000e+00, s: 5.1394e-03, l1_reg: 3.3386e+02, l2_reg: 6.6795e+01\n",
      "2020/09/15, 11:25:44, Iteration: 5700, Train Loss: 1.6088e-01, c1: 2.0737e-02, c2: 5.3257e-04, d1: 1.3481e-02, d2: 0.0000e+00, s: 3.7253e-03, l1_reg: 3.3372e+02, l2_reg: 6.6807e+01\n",
      "2020/09/15, 11:25:45, Iteration: 5800, Train Loss: 1.9224e-01, c1: 2.5021e-02, c2: 2.4988e-03, d1: 1.4223e-02, d2: 0.0000e+00, s: 3.3917e-03, l1_reg: 3.3361e+02, l2_reg: 6.6801e+01\n",
      "2020/09/15, 11:25:46, Iteration: 5900, Train Loss: 2.2166e-01, c1: 3.1937e-02, c2: 2.1247e-03, d1: 1.6847e-02, d2: 0.0000e+00, s: 4.5303e-03, l1_reg: 3.3345e+02, l2_reg: 6.6800e+01\n",
      "2020/09/15, 11:25:47, Iteration: 6000, Train Loss: 1.5985e-01, c1: 2.3459e-02, c2: 8.6026e-04, d1: 1.2779e-02, d2: 0.0000e+00, s: 3.4566e-03, l1_reg: 3.3337e+02, l2_reg: 6.6799e+01\n",
      "2020/09/15, 11:25:47, Iteration: 6100, Train Loss: 1.7007e-01, c1: 3.1941e-02, c2: 1.0694e-03, d1: 1.2743e-02, d2: 0.0000e+00, s: 3.3282e-03, l1_reg: 3.3324e+02, l2_reg: 6.6790e+01\n",
      "2020/09/15, 11:25:48, Iteration: 6200, Train Loss: 1.7637e-01, c1: 1.5877e-02, c2: 2.6477e-03, d1: 1.3402e-02, d2: 0.0000e+00, s: 3.3257e-03, l1_reg: 3.3311e+02, l2_reg: 6.6801e+01\n",
      "2020/09/15, 11:25:49, Iteration: 6300, Train Loss: 1.5317e-01, c1: 1.3132e-02, c2: 5.3531e-04, d1: 1.3468e-02, d2: 0.0000e+00, s: 3.2555e-03, l1_reg: 3.3300e+02, l2_reg: 6.6759e+01\n",
      "2020/09/15, 11:25:50, Iteration: 6400, Train Loss: 1.8743e-01, c1: 3.1370e-02, c2: 1.4623e-03, d1: 1.4143e-02, d2: 0.0000e+00, s: 3.8877e-03, l1_reg: 3.3292e+02, l2_reg: 6.6803e+01\n",
      "2020/09/15, 11:25:51, Iteration: 6500, Train Loss: 2.0333e-01, c1: 1.8915e-02, c2: 4.7152e-03, d1: 1.3726e-02, d2: 0.0000e+00, s: 3.3856e-03, l1_reg: 3.3278e+02, l2_reg: 6.6779e+01\n",
      "2020/09/15, 11:25:51, Iteration: 6600, Train Loss: 1.5048e-01, c1: 1.0141e-02, c2: 1.4774e-03, d1: 1.2556e-02, d2: 0.0000e+00, s: 3.8698e-03, l1_reg: 3.3264e+02, l2_reg: 6.6684e+01\n",
      "2020/09/15, 11:25:52, Iteration: 6700, Train Loss: 1.9440e-01, c1: 1.9881e-02, c2: 2.7628e-03, d1: 1.4689e-02, d2: 0.0000e+00, s: 3.8005e-03, l1_reg: 3.3251e+02, l2_reg: 6.6618e+01\n",
      "2020/09/15, 11:25:53, Iteration: 6800, Train Loss: 1.5274e-01, c1: 1.0398e-02, c2: 1.9888e-03, d1: 1.2245e-02, d2: 0.0000e+00, s: 3.1004e-03, l1_reg: 3.3241e+02, l2_reg: 6.6647e+01\n",
      "2020/09/15, 11:25:54, Iteration: 6900, Train Loss: 1.9743e-01, c1: 3.0519e-02, c2: 1.9248e-03, d1: 1.4766e-02, d2: 0.0000e+00, s: 3.2684e-03, l1_reg: 3.3223e+02, l2_reg: 6.6559e+01\n",
      "2020/09/15, 11:25:55, Iteration: 7000, Train Loss: 1.6230e-01, c1: 1.6393e-02, c2: 7.6407e-04, d1: 1.3826e-02, d2: 0.0000e+00, s: 2.8741e-03, l1_reg: 3.3207e+02, l2_reg: 6.6548e+01\n",
      "2020/09/15, 11:25:55, Iteration: 7100, Train Loss: 1.6554e-01, c1: 1.0167e-02, c2: 1.7661e-03, d1: 1.3771e-02, d2: 0.0000e+00, s: 3.0039e-03, l1_reg: 3.3193e+02, l2_reg: 6.6580e+01\n",
      "2020/09/15, 11:25:56, Iteration: 7200, Train Loss: 1.7988e-01, c1: 3.5325e-02, c2: 5.6282e-04, d1: 1.3893e-02, d2: 0.0000e+00, s: 3.3499e-03, l1_reg: 3.3179e+02, l2_reg: 6.6525e+01\n",
      "2020/09/15, 11:25:57, Iteration: 7300, Train Loss: 2.0556e-01, c1: 3.5941e-02, c2: 1.2925e-03, d1: 1.5669e-02, d2: 0.0000e+00, s: 3.9801e-03, l1_reg: 3.3170e+02, l2_reg: 6.6606e+01\n",
      "2020/09/15, 11:25:58, Iteration: 7400, Train Loss: 1.5594e-01, c1: 2.1000e-02, c2: 6.8180e-04, d1: 1.2812e-02, d2: 0.0000e+00, s: 2.8392e-03, l1_reg: 3.3159e+02, l2_reg: 6.6576e+01\n",
      "2020/09/15, 11:25:59, Iteration: 7500, Train Loss: 2.3019e-01, c1: 3.3607e-02, c2: 2.7755e-03, d1: 1.6883e-02, d2: 0.0000e+00, s: 3.1611e-03, l1_reg: 3.3141e+02, l2_reg: 6.6535e+01\n",
      "2020/09/15, 11:26:00, Iteration: 7600, Train Loss: 1.7727e-01, c1: 2.2468e-02, c2: 1.8719e-03, d1: 1.3608e-02, d2: 0.0000e+00, s: 2.7286e-03, l1_reg: 3.3128e+02, l2_reg: 6.6571e+01\n",
      "2020/09/15, 11:26:00, Iteration: 7700, Train Loss: 1.5596e-01, c1: 1.7424e-02, c2: 1.0712e-03, d1: 1.2782e-02, d2: 0.0000e+00, s: 4.1208e-03, l1_reg: 3.3111e+02, l2_reg: 6.6484e+01\n",
      "2020/09/15, 11:26:01, Iteration: 7800, Train Loss: 1.9675e-01, c1: 2.0112e-02, c2: 3.5989e-03, d1: 1.4064e-02, d2: 0.0000e+00, s: 2.6999e-03, l1_reg: 3.3096e+02, l2_reg: 6.6355e+01\n",
      "2020/09/15, 11:26:02, Iteration: 7900, Train Loss: 1.5946e-01, c1: 9.2680e-03, c2: 1.1535e-03, d1: 1.3866e-02, d2: 0.0000e+00, s: 2.8335e-03, l1_reg: 3.3083e+02, l2_reg: 6.6347e+01\n",
      "2020/09/15, 11:26:03, Iteration: 8000, Train Loss: 1.8346e-01, c1: 2.7447e-02, c2: 2.3380e-03, d1: 1.3263e-02, d2: 0.0000e+00, s: 4.2883e-03, l1_reg: 3.3072e+02, l2_reg: 6.6324e+01\n",
      "2020/09/15, 11:26:04, Iteration: 8100, Train Loss: 1.5792e-01, c1: 1.5856e-02, c2: 1.5032e-03, d1: 1.2703e-02, d2: 0.0000e+00, s: 3.1098e-03, l1_reg: 3.3057e+02, l2_reg: 6.6355e+01\n",
      "2020/09/15, 11:26:04, Iteration: 8200, Train Loss: 1.7391e-01, c1: 2.7240e-02, c2: 2.3239e-03, d1: 1.2343e-02, d2: 0.0000e+00, s: 2.4029e-03, l1_reg: 3.3043e+02, l2_reg: 6.6390e+01\n",
      "2020/09/15, 11:26:05, Iteration: 8300, Train Loss: 1.6419e-01, c1: 1.2093e-02, c2: 1.0962e-03, d1: 1.4113e-02, d2: 0.0000e+00, s: 2.7952e-03, l1_reg: 3.3029e+02, l2_reg: 6.6333e+01\n",
      "2020/09/15, 11:26:06, Iteration: 8400, Train Loss: 1.6648e-01, c1: 1.7973e-02, c2: 2.2363e-03, d1: 1.2614e-02, d2: 0.0000e+00, s: 2.4814e-03, l1_reg: 3.3011e+02, l2_reg: 6.6326e+01\n",
      "2020/09/15, 11:26:07, Iteration: 8500, Train Loss: 1.6770e-01, c1: 2.4149e-02, c2: 8.4520e-04, d1: 1.3510e-02, d2: 0.0000e+00, s: 2.9948e-03, l1_reg: 3.3001e+02, l2_reg: 6.6321e+01\n",
      "2020/09/15, 11:26:08, Iteration: 8600, Train Loss: 1.5880e-01, c1: 1.1958e-02, c2: 1.4499e-03, d1: 1.3234e-02, d2: 0.0000e+00, s: 4.1279e-03, l1_reg: 3.2988e+02, l2_reg: 6.6351e+01\n",
      "2020/09/15, 11:26:08, Iteration: 8700, Train Loss: 1.6130e-01, c1: 1.5257e-02, c2: 1.2831e-03, d1: 1.3321e-02, d2: 0.0000e+00, s: 2.9256e-03, l1_reg: 3.2976e+02, l2_reg: 6.6305e+01\n",
      "2020/09/15, 11:26:09, Iteration: 8800, Train Loss: 1.6899e-01, c1: 1.1716e-02, c2: 1.2957e-03, d1: 1.4431e-02, d2: 0.0000e+00, s: 2.7389e-03, l1_reg: 3.2962e+02, l2_reg: 6.6249e+01\n",
      "2020/09/15, 11:26:10, Iteration: 8900, Train Loss: 1.7840e-01, c1: 2.2032e-02, c2: 1.4983e-03, d1: 1.4139e-02, d2: 0.0000e+00, s: 3.7621e-03, l1_reg: 3.2948e+02, l2_reg: 6.6309e+01\n",
      "2020/09/15, 11:26:11, Iteration: 9000, Train Loss: 1.5043e-01, c1: 3.9834e-03, c2: 1.4709e-03, d1: 1.3174e-02, d2: 0.0000e+00, s: 2.7263e-03, l1_reg: 3.2935e+02, l2_reg: 6.6330e+01\n",
      "2020/09/15, 11:26:12, Iteration: 9100, Train Loss: 1.8313e-01, c1: 2.1399e-02, c2: 2.0490e-03, d1: 1.4124e-02, d2: 0.0000e+00, s: 1.9428e-03, l1_reg: 3.2918e+02, l2_reg: 6.6268e+01\n",
      "2020/09/15, 11:26:12, Iteration: 9200, Train Loss: 1.7392e-01, c1: 4.4671e-03, c2: 2.1605e-03, d1: 1.4784e-02, d2: 0.0000e+00, s: 3.3471e-03, l1_reg: 3.2904e+02, l2_reg: 6.6350e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/15, 11:26:13, Iteration: 9300, Train Loss: 1.5355e-01, c1: 6.0989e-03, c2: 1.2654e-03, d1: 1.3479e-02, d2: 0.0000e+00, s: 3.7145e-03, l1_reg: 3.2883e+02, l2_reg: 6.6295e+01\n",
      "2020/09/15, 11:26:14, Iteration: 9400, Train Loss: 1.5363e-01, c1: 9.7342e-03, c2: 1.9068e-03, d1: 1.2483e-02, d2: 0.0000e+00, s: 3.0894e-03, l1_reg: 3.2874e+02, l2_reg: 6.6371e+01\n",
      "2020/09/15, 11:26:15, Iteration: 9500, Train Loss: 2.2177e-01, c1: 2.4418e-02, c2: 3.1534e-03, d1: 1.6581e-02, d2: 0.0000e+00, s: 2.0828e-03, l1_reg: 3.2845e+02, l2_reg: 6.6313e+01\n",
      "2020/09/15, 11:26:16, Iteration: 9600, Train Loss: 1.5633e-01, c1: 6.2743e-03, c2: 1.8273e-03, d1: 1.3178e-02, d2: 0.0000e+00, s: 3.8755e-03, l1_reg: 3.2830e+02, l2_reg: 6.6279e+01\n",
      "2020/09/15, 11:26:16, Iteration: 9700, Train Loss: 2.0851e-01, c1: 1.8120e-02, c2: 2.5131e-03, d1: 1.6525e-02, d2: 0.0000e+00, s: 2.1285e-03, l1_reg: 3.2820e+02, l2_reg: 6.6305e+01\n",
      "2020/09/15, 11:26:17, Iteration: 9800, Train Loss: 1.6418e-01, c1: 1.4915e-02, c2: 1.2567e-03, d1: 1.3669e-02, d2: 0.0000e+00, s: 2.2725e-03, l1_reg: 3.2807e+02, l2_reg: 6.6310e+01\n",
      "2020/09/15, 11:26:18, Iteration: 9900, Train Loss: 1.8424e-01, c1: 1.9589e-02, c2: 1.5606e-03, d1: 1.4904e-02, d2: 0.0000e+00, s: 2.6183e-03, l1_reg: 3.2785e+02, l2_reg: 6.6162e+01\n",
      "2020/09/15, 11:26:19, Iteration: 10000, Train Loss: 1.6172e-01, c1: 1.5243e-02, c2: 1.2831e-03, d1: 1.3364e-02, d2: 0.0000e+00, s: 1.7287e-03, l1_reg: 3.2773e+02, l2_reg: 6.6120e+01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "import jax\n",
    "import jax.numpy as jnp\n",
    "\n",
    "import os, sys\n",
    "sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(\"./\")))))\n",
    "\n",
    "from training import Agent\n",
    "from loss import params, loss_fn, evaluate_fn\n",
    "from data import generate_dataset, generate_batch_fn\n",
    "\n",
    "import config\n",
    "config.n_data = {\n",
    "    \"i\": 100,\n",
    "    \"b\": 100,\n",
    "    \"cx\": 501,\n",
    "    \"ct\": 201,\n",
    "    \"dx\": 100,\n",
    "    \"dt\": 600,\n",
    "}\n",
    "config.batch_size = {\n",
    "    \"dirichlet\": (config.n_data[\"i\"]+2*config.n_data[\"b\"]+config.n_data[\"dx\"]*config.n_data[\"dt\"]) // 10,\n",
    "    \"collocation\": (config.n_data[\"dx\"]*config.n_data[\"dt\"]+config.n_data[\"cx\"]*config.n_data[\"ct\"]) // 10,\n",
    "}\n",
    "config.iterations = 10000\n",
    "config.print_every = 100\n",
    "config.lr = 1e-3\n",
    "config.weights = {\n",
    "\t\"c1\": 1.0,\n",
    "\t\"c2\": 10.0,\n",
    "\t\"d1\": 10.0,\n",
    "\t\"d2\": 10.0,\n",
    "\t\"l1\": 1e-8,\n",
    "\t\"l2\": 1e-8,\n",
    "    \"s\": 1e-8\n",
    "}\n",
    "\n",
    "datasets = generate_dataset(config.n_data[\"i\"], config.n_data[\"b\"], config.n_data[\"cx\"], config.n_data[\"ct\"], config.n_data[\"dx\"], config.n_data[\"dt\"])\n",
    "batch_fn, evaluate_batch_fn = generate_batch_fn(config.key, config.batch_size, *datasets, config.weights)\n",
    "\n",
    "agent = Agent(params, loss_fn, evaluate_fn, \"models/{}\".format(config.NAME))\n",
    "agent.compile(config.optimizer, config.lr)\n",
    "agent.train(config.iterations, batch_fn, evaluate_batch_fn, config.print_every, config.save_every, config.loss_names, config.log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/15, 11:26:27, Iteration: 11000, Train Loss: 1.5337e-01, c1: 1.0342e-02, c2: 1.6116e-03, d1: 1.2691e-02, d2: 0.0000e+00, s: 2.3720e-03, l1_reg: 3.2632e+02, l2_reg: 6.5880e+01\n",
      "2020/09/15, 11:26:35, Iteration: 12000, Train Loss: 1.6849e-01, c1: 1.9783e-02, c2: 1.1468e-03, d1: 1.3724e-02, d2: 0.0000e+00, s: 1.5530e-03, l1_reg: 3.2496e+02, l2_reg: 6.5404e+01\n",
      "2020/09/15, 11:26:43, Iteration: 13000, Train Loss: 1.6386e-01, c1: 2.4198e-02, c2: 1.2063e-03, d1: 1.2760e-02, d2: 0.0000e+00, s: 1.9544e-03, l1_reg: 3.2365e+02, l2_reg: 6.4986e+01\n",
      "2020/09/15, 11:26:51, Iteration: 14000, Train Loss: 1.7604e-01, c1: 7.8530e-03, c2: 1.1897e-03, d1: 1.5628e-02, d2: 0.0000e+00, s: 1.9382e-03, l1_reg: 3.2253e+02, l2_reg: 6.4770e+01\n",
      "2020/09/15, 11:26:59, Iteration: 15000, Train Loss: 1.5693e-01, c1: 9.9926e-03, c2: 5.3674e-04, d1: 1.4157e-02, d2: 0.0000e+00, s: 1.3050e-03, l1_reg: 3.2111e+02, l2_reg: 6.4272e+01\n",
      "2020/09/15, 11:27:08, Iteration: 16000, Train Loss: 1.6133e-01, c1: 1.7789e-02, c2: 1.2156e-03, d1: 1.3138e-02, d2: 0.0000e+00, s: 7.7968e-04, l1_reg: 3.1971e+02, l2_reg: 6.3806e+01\n",
      "2020/09/15, 11:27:16, Iteration: 17000, Train Loss: 1.7033e-01, c1: 2.6415e-02, c2: 8.4311e-04, d1: 1.3548e-02, d2: 0.0000e+00, s: 8.5037e-04, l1_reg: 3.1732e+02, l2_reg: 6.3304e+01\n",
      "2020/09/15, 11:27:24, Iteration: 18000, Train Loss: 1.5196e-01, c1: 7.3737e-03, c2: 8.5915e-04, d1: 1.3599e-02, d2: 0.0000e+00, s: 7.8373e-04, l1_reg: 3.1577e+02, l2_reg: 6.3005e+01\n",
      "2020/09/15, 11:27:32, Iteration: 19000, Train Loss: 1.5099e-01, c1: 9.8268e-03, c2: 7.0426e-04, d1: 1.3412e-02, d2: 0.0000e+00, s: 3.7730e-04, l1_reg: 3.1266e+02, l2_reg: 6.2364e+01\n",
      "2020/09/15, 11:27:40, Iteration: 20000, Train Loss: 1.4754e-01, c1: 1.2060e-02, c2: 2.8029e-04, d1: 1.3268e-02, d2: 0.0000e+00, s: 4.0093e-04, l1_reg: 3.1078e+02, l2_reg: 6.2008e+01\n",
      "2020/09/15, 11:27:48, Iteration: 21000, Train Loss: 1.6293e-01, c1: 3.3607e-03, c2: 9.1794e-04, d1: 1.5038e-02, d2: 0.0000e+00, s: 2.5372e-04, l1_reg: 3.0838e+02, l2_reg: 6.1598e+01\n",
      "2020/09/15, 11:27:57, Iteration: 22000, Train Loss: 1.4872e-01, c1: 1.2904e-02, c2: 4.3299e-04, d1: 1.3148e-02, d2: 0.0000e+00, s: 2.1950e-04, l1_reg: 3.0594e+02, l2_reg: 6.1426e+01\n",
      "2020/09/15, 11:28:05, Iteration: 23000, Train Loss: 1.6684e-01, c1: 2.0192e-02, c2: 1.6605e-03, d1: 1.3004e-02, d2: 0.0000e+00, s: 1.6495e-04, l1_reg: 3.0388e+02, l2_reg: 6.1167e+01\n",
      "2020/09/15, 11:28:13, Iteration: 24000, Train Loss: 1.4904e-01, c1: 6.1080e-03, c2: 1.3249e-03, d1: 1.2968e-02, d2: 0.0000e+00, s: 1.3422e-04, l1_reg: 3.0134e+02, l2_reg: 6.0879e+01\n",
      "2020/09/15, 11:28:21, Iteration: 25000, Train Loss: 1.3728e-01, c1: 3.9632e-03, c2: 9.4979e-04, d1: 1.2382e-02, d2: 0.0000e+00, s: 1.0078e-04, l1_reg: 2.9809e+02, l2_reg: 6.0590e+01\n",
      "2020/09/15, 11:28:29, Iteration: 26000, Train Loss: 1.5361e-01, c1: 8.7383e-03, c2: 5.9839e-04, d1: 1.3889e-02, d2: 0.0000e+00, s: 3.8789e-05, l1_reg: 2.9597e+02, l2_reg: 6.0455e+01\n",
      "2020/09/15, 11:28:37, Iteration: 27000, Train Loss: 1.5552e-01, c1: 1.0885e-02, c2: 1.3786e-03, d1: 1.3085e-02, d2: 0.0000e+00, s: 1.5660e-05, l1_reg: 2.9249e+02, l2_reg: 6.0191e+01\n",
      "2020/09/15, 11:28:45, Iteration: 28000, Train Loss: 1.4913e-01, c1: 5.8784e-03, c2: 4.4245e-04, d1: 1.3882e-02, d2: 0.0000e+00, s: 1.8495e-05, l1_reg: 2.8943e+02, l2_reg: 5.9759e+01\n",
      "2020/09/15, 11:28:53, Iteration: 29000, Train Loss: 1.5180e-01, c1: 5.9077e-03, c2: 8.1710e-04, d1: 1.3771e-02, d2: 0.0000e+00, s: 8.6395e-07, l1_reg: 2.8539e+02, l2_reg: 5.9234e+01\n",
      "2020/09/15, 11:29:01, Iteration: 30000, Train Loss: 1.3450e-01, c1: 1.9520e-03, c2: 9.7673e-04, d1: 1.2278e-02, d2: 0.0000e+00, s: 1.0906e-05, l1_reg: 2.8192e+02, l2_reg: 5.8882e+01\n",
      "2020/09/15, 11:29:09, Iteration: 31000, Train Loss: 1.5163e-01, c1: 5.0893e-03, c2: 1.7756e-03, d1: 1.2879e-02, d2: 0.0000e+00, s: 3.2427e-07, l1_reg: 2.7944e+02, l2_reg: 5.8644e+01\n",
      "2020/09/15, 11:29:18, Iteration: 32000, Train Loss: 1.5188e-01, c1: 7.8417e-03, c2: 3.1003e-04, d1: 1.4093e-02, d2: 0.0000e+00, s: 2.2613e-07, l1_reg: 2.7747e+02, l2_reg: 5.8631e+01\n",
      "2020/09/15, 11:29:26, Iteration: 33000, Train Loss: 1.4387e-01, c1: 5.0045e-03, c2: 4.8400e-04, d1: 1.3402e-02, d2: 0.0000e+00, s: 1.2414e-06, l1_reg: 2.7344e+02, l2_reg: 5.8084e+01\n",
      "2020/09/15, 11:29:34, Iteration: 34000, Train Loss: 1.5001e-01, c1: 1.5825e-02, c2: 7.1961e-04, d1: 1.2699e-02, d2: 0.0000e+00, s: 6.9066e-08, l1_reg: 2.6955e+02, l2_reg: 5.7556e+01\n",
      "2020/09/15, 11:29:42, Iteration: 35000, Train Loss: 1.4300e-01, c1: 6.1600e-03, c2: 7.9010e-04, d1: 1.2894e-02, d2: 0.0000e+00, s: 1.9140e-05, l1_reg: 2.6592e+02, l2_reg: 5.7296e+01\n",
      "2020/09/15, 11:29:50, Iteration: 36000, Train Loss: 1.2853e-01, c1: 1.8413e-03, c2: 2.7549e-04, d1: 1.2393e-02, d2: 0.0000e+00, s: 3.4327e-07, l1_reg: 2.6250e+02, l2_reg: 5.6964e+01\n",
      "2020/09/15, 11:29:58, Iteration: 37000, Train Loss: 1.4486e-01, c1: 4.4521e-03, c2: 1.2528e-03, d1: 1.2788e-02, d2: 0.0000e+00, s: 1.2792e-07, l1_reg: 2.5957e+02, l2_reg: 5.6849e+01\n",
      "2020/09/15, 11:30:06, Iteration: 38000, Train Loss: 1.3132e-01, c1: 5.8034e-03, c2: 2.6197e-04, d1: 1.2290e-02, d2: 0.0000e+00, s: 1.2105e-05, l1_reg: 2.5700e+02, l2_reg: 5.6430e+01\n",
      "2020/09/15, 11:30:14, Iteration: 39000, Train Loss: 1.4568e-01, c1: 8.3258e-03, c2: 8.3192e-04, d1: 1.2903e-02, d2: 0.0000e+00, s: 1.4785e-05, l1_reg: 2.5461e+02, l2_reg: 5.6471e+01\n",
      "2020/09/15, 11:30:23, Iteration: 40000, Train Loss: 1.5582e-01, c1: 1.9547e-02, c2: 8.3288e-04, d1: 1.2794e-02, d2: 0.0000e+00, s: 2.1708e-07, l1_reg: 2.5010e+02, l2_reg: 5.6094e+01\n",
      "2020/09/15, 11:30:31, Iteration: 41000, Train Loss: 1.3496e-01, c1: 2.5055e-03, c2: 7.6616e-04, d1: 1.2478e-02, d2: 0.0000e+00, s: 2.2858e-07, l1_reg: 2.4938e+02, l2_reg: 5.6322e+01\n",
      "2020/09/15, 11:30:39, Iteration: 42000, Train Loss: 1.4166e-01, c1: 1.1884e-02, c2: 7.0264e-04, d1: 1.2275e-02, d2: 0.0000e+00, s: 1.0569e-07, l1_reg: 2.4783e+02, l2_reg: 5.6547e+01\n",
      "2020/09/15, 11:30:47, Iteration: 43000, Train Loss: 1.4050e-01, c1: 1.0348e-02, c2: 8.1411e-04, d1: 1.2200e-02, d2: 0.0000e+00, s: 4.6475e-07, l1_reg: 2.4399e+02, l2_reg: 5.6366e+01\n",
      "2020/09/15, 11:30:55, Iteration: 44000, Train Loss: 1.3010e-01, c1: 1.9253e-03, c2: 6.7325e-04, d1: 1.2144e-02, d2: 0.0000e+00, s: 1.8722e-07, l1_reg: 2.4160e+02, l2_reg: 5.6392e+01\n",
      "2020/09/15, 11:31:03, Iteration: 45000, Train Loss: 1.4040e-01, c1: 1.0084e-02, c2: 4.6453e-04, d1: 1.2567e-02, d2: 0.0000e+00, s: 1.2481e-07, l1_reg: 2.3812e+02, l2_reg: 5.6196e+01\n",
      "2020/09/15, 11:31:11, Iteration: 46000, Train Loss: 1.6114e-01, c1: 8.7301e-03, c2: 2.6730e-03, d1: 1.2568e-02, d2: 0.0000e+00, s: 9.2540e-08, l1_reg: 2.3658e+02, l2_reg: 5.6322e+01\n",
      "2020/09/15, 11:31:19, Iteration: 47000, Train Loss: 1.4953e-01, c1: 6.9010e-03, c2: 4.7370e-04, d1: 1.3789e-02, d2: 0.0000e+00, s: 2.5909e-08, l1_reg: 2.3413e+02, l2_reg: 5.6191e+01\n",
      "2020/09/15, 11:31:27, Iteration: 48000, Train Loss: 1.3729e-01, c1: 5.6595e-03, c2: 9.9080e-04, d1: 1.2172e-02, d2: 0.0000e+00, s: 4.0236e-08, l1_reg: 2.3249e+02, l2_reg: 5.6191e+01\n",
      "2020/09/15, 11:31:36, Iteration: 49000, Train Loss: 1.3494e-01, c1: 5.7790e-03, c2: 4.8839e-04, d1: 1.2428e-02, d2: 0.0000e+00, s: 1.2250e-09, l1_reg: 2.3066e+02, l2_reg: 5.6241e+01\n",
      "2020/09/15, 11:31:44, Iteration: 50000, Train Loss: 1.3414e-01, c1: 3.4464e-03, c2: 8.5193e-04, d1: 1.2217e-02, d2: 0.0000e+00, s: 2.7023e-09, l1_reg: 2.2732e+02, l2_reg: 5.6154e+01\n",
      "2020/09/15, 11:31:52, Iteration: 51000, Train Loss: 1.4324e-01, c1: 5.6203e-03, c2: 1.3033e-03, d1: 1.2458e-02, d2: 0.0000e+00, s: 5.2331e-09, l1_reg: 2.2608e+02, l2_reg: 5.6205e+01\n",
      "2020/09/15, 11:32:00, Iteration: 52000, Train Loss: 1.5301e-01, c1: 2.2241e-02, c2: 6.1996e-04, d1: 1.2456e-02, d2: 0.0000e+00, s: 5.3171e-10, l1_reg: 2.2513e+02, l2_reg: 5.6579e+01\n",
      "2020/09/15, 11:32:08, Iteration: 53000, Train Loss: 1.4733e-01, c1: 4.2399e-03, c2: 9.1974e-04, d1: 1.3389e-02, d2: 0.0000e+00, s: 1.4006e-09, l1_reg: 2.2317e+02, l2_reg: 5.6538e+01\n",
      "2020/09/15, 11:32:16, Iteration: 54000, Train Loss: 1.3769e-01, c1: 7.3616e-03, c2: 3.7384e-04, d1: 1.2658e-02, d2: 0.0000e+00, s: 2.0696e-11, l1_reg: 2.2225e+02, l2_reg: 5.6836e+01\n",
      "2020/09/15, 11:32:25, Iteration: 55000, Train Loss: 1.5322e-01, c1: 3.1287e-03, c2: 2.8045e-03, d1: 1.2204e-02, d2: 0.0000e+00, s: 1.2229e-10, l1_reg: 2.1916e+02, l2_reg: 5.6723e+01\n",
      "2020/09/15, 11:32:33, Iteration: 56000, Train Loss: 1.3729e-01, c1: 2.4511e-03, c2: 7.5601e-04, d1: 1.2727e-02, d2: 0.0000e+00, s: 7.7040e-10, l1_reg: 2.1783e+02, l2_reg: 5.6894e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/15, 11:32:41, Iteration: 57000, Train Loss: 1.3122e-01, c1: 2.2719e-03, c2: 5.8701e-04, d1: 1.2307e-02, d2: 0.0000e+00, s: 2.2832e-10, l1_reg: 2.1629e+02, l2_reg: 5.6823e+01\n",
      "2020/09/15, 11:32:49, Iteration: 58000, Train Loss: 1.3348e-01, c1: 6.0530e-03, c2: 5.8408e-04, d1: 1.2158e-02, d2: 0.0000e+00, s: 2.2695e-11, l1_reg: 2.1350e+02, l2_reg: 5.7038e+01\n",
      "2020/09/15, 11:32:57, Iteration: 59000, Train Loss: 1.3544e-01, c1: 2.6818e-03, c2: 3.4559e-04, d1: 1.2930e-02, d2: 0.0000e+00, s: 2.8697e-10, l1_reg: 2.1152e+02, l2_reg: 5.6835e+01\n",
      "2020/09/15, 11:33:05, Iteration: 60000, Train Loss: 1.3593e-01, c1: 7.3576e-03, c2: 3.0573e-04, d1: 1.2552e-02, d2: 0.0000e+00, s: 6.4612e-09, l1_reg: 2.0951e+02, l2_reg: 5.6931e+01\n",
      "2020/09/15, 11:33:13, Iteration: 61000, Train Loss: 1.4128e-01, c1: 4.8041e-03, c2: 1.0007e-03, d1: 1.2646e-02, d2: 0.0000e+00, s: 2.0829e-12, l1_reg: 2.0944e+02, l2_reg: 5.7190e+01\n",
      "2020/09/15, 11:33:21, Iteration: 62000, Train Loss: 1.3342e-01, c1: 6.0840e-03, c2: 4.6244e-04, d1: 1.2271e-02, d2: 0.0000e+00, s: 8.0189e-13, l1_reg: 2.0861e+02, l2_reg: 5.7253e+01\n",
      "2020/09/15, 11:33:29, Iteration: 63000, Train Loss: 1.3203e-01, c1: 3.1034e-03, c2: 7.7345e-04, d1: 1.2119e-02, d2: 0.0000e+00, s: 1.4020e-12, l1_reg: 2.0816e+02, l2_reg: 5.7518e+01\n",
      "2020/09/15, 11:33:38, Iteration: 64000, Train Loss: 1.3992e-01, c1: 6.9909e-03, c2: 8.2007e-04, d1: 1.2472e-02, d2: 0.0000e+00, s: 2.9195e-13, l1_reg: 2.0457e+02, l2_reg: 5.7742e+01\n",
      "2020/09/15, 11:33:46, Iteration: 65000, Train Loss: 1.3969e-01, c1: 4.7679e-03, c2: 9.9917e-04, d1: 1.2493e-02, d2: 0.0000e+00, s: 1.2502e-11, l1_reg: 2.0494e+02, l2_reg: 5.7942e+01\n",
      "2020/09/15, 11:33:54, Iteration: 66000, Train Loss: 1.3926e-01, c1: 8.8382e-03, c2: 8.9485e-04, d1: 1.2147e-02, d2: 0.0000e+00, s: 4.4190e-12, l1_reg: 2.0285e+02, l2_reg: 5.8142e+01\n",
      "2020/09/15, 11:34:02, Iteration: 67000, Train Loss: 1.3871e-01, c1: 8.7490e-03, c2: 5.6370e-04, d1: 1.2432e-02, d2: 0.0000e+00, s: 5.7611e-11, l1_reg: 2.0149e+02, l2_reg: 5.8589e+01\n",
      "2020/09/15, 11:34:10, Iteration: 68000, Train Loss: 1.4490e-01, c1: 4.1986e-03, c2: 4.4240e-04, d1: 1.3627e-02, d2: 0.0000e+00, s: 1.4715e-10, l1_reg: 2.0104e+02, l2_reg: 5.8680e+01\n",
      "2020/09/15, 11:34:18, Iteration: 69000, Train Loss: 1.3148e-01, c1: 2.9952e-03, c2: 7.6758e-04, d1: 1.2080e-02, d2: 0.0000e+00, s: 1.6569e-11, l1_reg: 1.9935e+02, l2_reg: 5.8829e+01\n",
      "2020/09/15, 11:34:26, Iteration: 70000, Train Loss: 1.3211e-01, c1: 3.0314e-03, c2: 8.3611e-04, d1: 1.2071e-02, d2: 0.0000e+00, s: 3.7823e-11, l1_reg: 2.0150e+02, l2_reg: 5.8923e+01\n",
      "2020/09/15, 11:34:34, Iteration: 71000, Train Loss: 1.3590e-01, c1: 3.2559e-03, c2: 9.4670e-04, d1: 1.2317e-02, d2: 0.0000e+00, s: 4.8093e-12, l1_reg: 1.9806e+02, l2_reg: 5.9040e+01\n",
      "2020/09/15, 11:34:42, Iteration: 72000, Train Loss: 1.3726e-01, c1: 6.2667e-03, c2: 6.6260e-04, d1: 1.2436e-02, d2: 0.0000e+00, s: 2.4304e-09, l1_reg: 1.9852e+02, l2_reg: 5.9477e+01\n",
      "2020/09/15, 11:34:50, Iteration: 73000, Train Loss: 1.4000e-01, c1: 3.3131e-03, c2: 1.4084e-03, d1: 1.2261e-02, d2: 0.0000e+00, s: 6.3549e-10, l1_reg: 1.9668e+02, l2_reg: 5.9603e+01\n",
      "2020/09/15, 11:34:58, Iteration: 74000, Train Loss: 1.4184e-01, c1: 2.2410e-03, c2: 1.1063e-03, d1: 1.2854e-02, d2: 0.0000e+00, s: 1.2536e-11, l1_reg: 1.9915e+02, l2_reg: 5.9936e+01\n",
      "2020/09/15, 11:35:06, Iteration: 75000, Train Loss: 1.3309e-01, c1: 5.6234e-03, c2: 3.0322e-04, d1: 1.2444e-02, d2: 0.0000e+00, s: 1.4361e-08, l1_reg: 1.9612e+02, l2_reg: 6.0209e+01\n",
      "2020/09/15, 11:35:14, Iteration: 76000, Train Loss: 1.2989e-01, c1: 2.7249e-03, c2: 5.3044e-04, d1: 1.2186e-02, d2: 0.0000e+00, s: 1.4242e-09, l1_reg: 1.9488e+02, l2_reg: 6.0276e+01\n",
      "2020/09/15, 11:35:22, Iteration: 77000, Train Loss: 1.4185e-01, c1: 4.7287e-03, c2: 1.5355e-03, d1: 1.2177e-02, d2: 0.0000e+00, s: 1.6547e-11, l1_reg: 1.9503e+02, l2_reg: 6.0687e+01\n",
      "2020/09/15, 11:35:30, Iteration: 78000, Train Loss: 1.3379e-01, c1: 1.3911e-03, c2: 8.5569e-04, d1: 1.2384e-02, d2: 0.0000e+00, s: 9.5929e-11, l1_reg: 1.9481e+02, l2_reg: 6.0881e+01\n",
      "2020/09/15, 11:35:39, Iteration: 79000, Train Loss: 1.3329e-01, c1: 4.9346e-03, c2: 4.3392e-04, d1: 1.2401e-02, d2: 0.0000e+00, s: 2.2602e-08, l1_reg: 1.9363e+02, l2_reg: 6.1171e+01\n",
      "2020/09/15, 11:35:47, Iteration: 80000, Train Loss: 1.3664e-01, c1: 3.7398e-03, c2: 9.1257e-04, d1: 1.2377e-02, d2: 0.0000e+00, s: 4.6627e-11, l1_reg: 1.9541e+02, l2_reg: 6.1413e+01\n",
      "2020/09/15, 11:35:55, Iteration: 81000, Train Loss: 1.3048e-01, c1: 4.8115e-03, c2: 2.6983e-04, d1: 1.2297e-02, d2: 0.0000e+00, s: 7.3118e-10, l1_reg: 1.9445e+02, l2_reg: 6.1336e+01\n",
      "2020/09/15, 11:36:03, Iteration: 82000, Train Loss: 1.3265e-01, c1: 1.5820e-03, c2: 9.0237e-04, d1: 1.2204e-02, d2: 0.0000e+00, s: 1.7333e-13, l1_reg: 1.9108e+02, l2_reg: 6.1352e+01\n",
      "2020/09/15, 11:36:11, Iteration: 83000, Train Loss: 1.5405e-01, c1: 6.1694e-03, c2: 2.2481e-03, d1: 1.2539e-02, d2: 0.0000e+00, s: 3.7770e-11, l1_reg: 1.9075e+02, l2_reg: 6.1542e+01\n",
      "2020/09/15, 11:36:19, Iteration: 84000, Train Loss: 1.4613e-01, c1: 1.1227e-02, c2: 1.4139e-03, d1: 1.2076e-02, d2: 0.0000e+00, s: 1.7848e-10, l1_reg: 1.8961e+02, l2_reg: 6.1678e+01\n",
      "2020/09/15, 11:36:27, Iteration: 85000, Train Loss: 1.3873e-01, c1: 7.6218e-03, c2: 4.6791e-04, d1: 1.2643e-02, d2: 0.0000e+00, s: 6.7038e-11, l1_reg: 1.8671e+02, l2_reg: 6.1787e+01\n",
      "2020/09/15, 11:36:35, Iteration: 86000, Train Loss: 1.3697e-01, c1: 3.0271e-03, c2: 1.1206e-03, d1: 1.2273e-02, d2: 0.0000e+00, s: 3.4697e-13, l1_reg: 1.8669e+02, l2_reg: 6.1887e+01\n",
      "2020/09/15, 11:36:43, Iteration: 87000, Train Loss: 1.4534e-01, c1: 7.2393e-03, c2: 6.3128e-04, d1: 1.3178e-02, d2: 0.0000e+00, s: 5.3941e-12, l1_reg: 1.8586e+02, l2_reg: 6.2167e+01\n",
      "2020/09/15, 11:36:52, Iteration: 88000, Train Loss: 1.3987e-01, c1: 1.1442e-02, c2: 4.7931e-04, d1: 1.2363e-02, d2: 0.0000e+00, s: 6.0336e-12, l1_reg: 1.8681e+02, l2_reg: 6.2509e+01\n",
      "2020/09/15, 11:37:00, Iteration: 89000, Train Loss: 1.3889e-01, c1: 4.2257e-03, c2: 5.3743e-04, d1: 1.2928e-02, d2: 0.0000e+00, s: 2.1425e-11, l1_reg: 1.8476e+02, l2_reg: 6.2875e+01\n",
      "2020/09/15, 11:37:08, Iteration: 90000, Train Loss: 1.2522e-01, c1: 9.8371e-04, c2: 2.6993e-04, d1: 1.2153e-02, d2: 0.0000e+00, s: 3.1547e-10, l1_reg: 1.8699e+02, l2_reg: 6.3258e+01\n",
      "2020/09/15, 11:37:16, Iteration: 91000, Train Loss: 1.4219e-01, c1: 2.5384e-03, c2: 1.6786e-03, d1: 1.2286e-02, d2: 0.0000e+00, s: 3.3549e-12, l1_reg: 1.8546e+02, l2_reg: 6.3336e+01\n",
      "2020/09/15, 11:37:24, Iteration: 92000, Train Loss: 1.3113e-01, c1: 3.4434e-03, c2: 4.4278e-04, d1: 1.2326e-02, d2: 0.0000e+00, s: 8.2608e-11, l1_reg: 1.8356e+02, l2_reg: 6.3549e+01\n",
      "2020/09/15, 11:37:33, Iteration: 93000, Train Loss: 1.4638e-01, c1: 8.4263e-03, c2: 1.4746e-03, d1: 1.2321e-02, d2: 0.0000e+00, s: 8.5829e-12, l1_reg: 1.8218e+02, l2_reg: 6.4082e+01\n",
      "2020/09/15, 11:37:40, Iteration: 94000, Train Loss: 1.3441e-01, c1: 3.7893e-03, c2: 8.1300e-04, d1: 1.2249e-02, d2: 0.0000e+00, s: 2.0747e-11, l1_reg: 1.8389e+02, l2_reg: 6.4591e+01\n",
      "2020/09/15, 11:37:49, Iteration: 95000, Train Loss: 1.3676e-01, c1: 3.7791e-03, c2: 7.1483e-04, d1: 1.2583e-02, d2: 0.0000e+00, s: 4.8655e-11, l1_reg: 1.8332e+02, l2_reg: 6.4865e+01\n",
      "2020/09/15, 11:37:57, Iteration: 96000, Train Loss: 1.4185e-01, c1: 2.2629e-03, c2: 1.2704e-03, d1: 1.2688e-02, d2: 0.0000e+00, s: 4.0147e-12, l1_reg: 1.8396e+02, l2_reg: 6.5243e+01\n",
      "2020/09/15, 11:38:05, Iteration: 97000, Train Loss: 1.3776e-01, c1: 7.3969e-03, c2: 6.5962e-04, d1: 1.2377e-02, d2: 0.0000e+00, s: 4.9378e-13, l1_reg: 1.8333e+02, l2_reg: 6.5797e+01\n",
      "2020/09/15, 11:38:13, Iteration: 98000, Train Loss: 1.2821e-01, c1: 1.2099e-03, c2: 5.1040e-04, d1: 1.2189e-02, d2: 0.0000e+00, s: 1.4900e-11, l1_reg: 1.8297e+02, l2_reg: 6.6060e+01\n",
      "2020/09/15, 11:38:21, Iteration: 99000, Train Loss: 1.4140e-01, c1: 1.7627e-03, c2: 1.4596e-03, d1: 1.2504e-02, d2: 0.0000e+00, s: 3.3190e-11, l1_reg: 1.8163e+02, l2_reg: 6.5981e+01\n",
      "2020/09/15, 11:38:29, Iteration: 100000, Train Loss: 1.3006e-01, c1: 1.9810e-03, c2: 4.4953e-04, d1: 1.2358e-02, d2: 0.0000e+00, s: 2.0284e-13, l1_reg: 1.8158e+02, l2_reg: 6.6652e+01\n",
      "2020/09/15, 11:38:37, Iteration: 101000, Train Loss: 1.3396e-01, c1: 5.6651e-03, c2: 7.3313e-04, d1: 1.2097e-02, d2: 0.0000e+00, s: 2.0554e-10, l1_reg: 1.8167e+02, l2_reg: 6.7067e+01\n",
      "2020/09/15, 11:38:45, Iteration: 102000, Train Loss: 1.3345e-01, c1: 5.0497e-03, c2: 3.2412e-04, d1: 1.2516e-02, d2: 0.0000e+00, s: 1.1771e-09, l1_reg: 1.8113e+02, l2_reg: 6.7630e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/15, 11:38:53, Iteration: 103000, Train Loss: 1.3604e-01, c1: 5.4116e-03, c2: 4.6019e-04, d1: 1.2603e-02, d2: 0.0000e+00, s: 2.1009e-12, l1_reg: 1.8086e+02, l2_reg: 6.7634e+01\n",
      "2020/09/15, 11:39:02, Iteration: 104000, Train Loss: 1.2397e-01, c1: 7.4872e-04, c2: 2.5174e-04, d1: 1.2070e-02, d2: 0.0000e+00, s: 1.0000e-12, l1_reg: 1.8081e+02, l2_reg: 6.8090e+01\n",
      "2020/09/15, 11:39:10, Iteration: 105000, Train Loss: 1.2645e-01, c1: 2.6267e-03, c2: 2.3623e-04, d1: 1.2146e-02, d2: 0.0000e+00, s: 4.6665e-10, l1_reg: 1.7830e+02, l2_reg: 6.8373e+01\n",
      "2020/09/15, 11:39:18, Iteration: 106000, Train Loss: 1.3556e-01, c1: 6.8455e-03, c2: 4.3208e-04, d1: 1.2439e-02, d2: 0.0000e+00, s: 1.4989e-11, l1_reg: 1.7920e+02, l2_reg: 6.9034e+01\n",
      "2020/09/15, 11:39:26, Iteration: 107000, Train Loss: 1.3041e-01, c1: 2.3955e-03, c2: 2.9929e-04, d1: 1.2502e-02, d2: 0.0000e+00, s: 2.6859e-10, l1_reg: 1.7740e+02, l2_reg: 6.9291e+01\n",
      "2020/09/15, 11:39:34, Iteration: 108000, Train Loss: 1.3645e-01, c1: 5.1485e-03, c2: 7.0930e-04, d1: 1.2420e-02, d2: 0.0000e+00, s: 6.5504e-11, l1_reg: 1.7814e+02, l2_reg: 6.9583e+01\n",
      "2020/09/15, 11:39:42, Iteration: 109000, Train Loss: 1.3396e-01, c1: 6.0220e-03, c2: 4.1928e-04, d1: 1.2375e-02, d2: 0.0000e+00, s: 9.7961e-09, l1_reg: 1.7843e+02, l2_reg: 7.0024e+01\n",
      "2020/09/15, 11:39:50, Iteration: 110000, Train Loss: 1.3261e-01, c1: 3.1376e-03, c2: 2.2304e-04, d1: 1.2724e-02, d2: 0.0000e+00, s: 1.3317e-09, l1_reg: 1.7637e+02, l2_reg: 7.0421e+01\n"
     ]
    }
   ],
   "source": [
    "config.iterations = 100000\n",
    "config.print_every = 1000\n",
    "agent.train(config.iterations, batch_fn, evaluate_batch_fn, config.print_every, config.save_every, config.loss_names, config.log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "/* Put everything inside the global mpl namespace */\n",
       "/* global mpl */\n",
       "window.mpl = {};\n",
       "\n",
       "mpl.get_websocket_type = function () {\n",
       "    if (typeof WebSocket !== 'undefined') {\n",
       "        return WebSocket;\n",
       "    } else if (typeof MozWebSocket !== 'undefined') {\n",
       "        return MozWebSocket;\n",
       "    } else {\n",
       "        alert(\n",
       "            'Your browser does not have WebSocket support. ' +\n",
       "                'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
       "                'Firefox 4 and 5 are also supported but you ' +\n",
       "                'have to enable WebSockets in about:config.'\n",
       "        );\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n",
       "    this.id = figure_id;\n",
       "\n",
       "    this.ws = websocket;\n",
       "\n",
       "    this.supports_binary = this.ws.binaryType !== undefined;\n",
       "\n",
       "    if (!this.supports_binary) {\n",
       "        var warnings = document.getElementById('mpl-warnings');\n",
       "        if (warnings) {\n",
       "            warnings.style.display = 'block';\n",
       "            warnings.textContent =\n",
       "                'This browser does not support binary websocket messages. ' +\n",
       "                'Performance may be slow.';\n",
       "        }\n",
       "    }\n",
       "\n",
       "    this.imageObj = new Image();\n",
       "\n",
       "    this.context = undefined;\n",
       "    this.message = undefined;\n",
       "    this.canvas = undefined;\n",
       "    this.rubberband_canvas = undefined;\n",
       "    this.rubberband_context = undefined;\n",
       "    this.format_dropdown = undefined;\n",
       "\n",
       "    this.image_mode = 'full';\n",
       "\n",
       "    this.root = document.createElement('div');\n",
       "    this.root.setAttribute('style', 'display: inline-block');\n",
       "    this._root_extra_style(this.root);\n",
       "\n",
       "    parent_element.appendChild(this.root);\n",
       "\n",
       "    this._init_header(this);\n",
       "    this._init_canvas(this);\n",
       "    this._init_toolbar(this);\n",
       "\n",
       "    var fig = this;\n",
       "\n",
       "    this.waiting = false;\n",
       "\n",
       "    this.ws.onopen = function () {\n",
       "        fig.send_message('supports_binary', { value: fig.supports_binary });\n",
       "        fig.send_message('send_image_mode', {});\n",
       "        if (mpl.ratio !== 1) {\n",
       "            fig.send_message('set_dpi_ratio', { dpi_ratio: mpl.ratio });\n",
       "        }\n",
       "        fig.send_message('refresh', {});\n",
       "    };\n",
       "\n",
       "    this.imageObj.onload = function () {\n",
       "        if (fig.image_mode === 'full') {\n",
       "            // Full images could contain transparency (where diff images\n",
       "            // almost always do), so we need to clear the canvas so that\n",
       "            // there is no ghosting.\n",
       "            fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
       "        }\n",
       "        fig.context.drawImage(fig.imageObj, 0, 0);\n",
       "    };\n",
       "\n",
       "    this.imageObj.onunload = function () {\n",
       "        fig.ws.close();\n",
       "    };\n",
       "\n",
       "    this.ws.onmessage = this._make_on_message_function(this);\n",
       "\n",
       "    this.ondownload = ondownload;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_header = function () {\n",
       "    var titlebar = document.createElement('div');\n",
       "    titlebar.classList =\n",
       "        'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n",
       "    var titletext = document.createElement('div');\n",
       "    titletext.classList = 'ui-dialog-title';\n",
       "    titletext.setAttribute(\n",
       "        'style',\n",
       "        'width: 100%; text-align: center; padding: 3px;'\n",
       "    );\n",
       "    titlebar.appendChild(titletext);\n",
       "    this.root.appendChild(titlebar);\n",
       "    this.header = titletext;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._init_canvas = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var canvas_div = (this.canvas_div = document.createElement('div'));\n",
       "    canvas_div.setAttribute(\n",
       "        'style',\n",
       "        'border: 1px solid #ddd;' +\n",
       "            'box-sizing: content-box;' +\n",
       "            'clear: both;' +\n",
       "            'min-height: 1px;' +\n",
       "            'min-width: 1px;' +\n",
       "            'outline: 0;' +\n",
       "            'overflow: hidden;' +\n",
       "            'position: relative;' +\n",
       "            'resize: both;'\n",
       "    );\n",
       "\n",
       "    function on_keyboard_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.key_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    canvas_div.addEventListener(\n",
       "        'keydown',\n",
       "        on_keyboard_event_closure('key_press')\n",
       "    );\n",
       "    canvas_div.addEventListener(\n",
       "        'keyup',\n",
       "        on_keyboard_event_closure('key_release')\n",
       "    );\n",
       "\n",
       "    this._canvas_extra_style(canvas_div);\n",
       "    this.root.appendChild(canvas_div);\n",
       "\n",
       "    var canvas = (this.canvas = document.createElement('canvas'));\n",
       "    canvas.classList.add('mpl-canvas');\n",
       "    canvas.setAttribute('style', 'box-sizing: content-box;');\n",
       "\n",
       "    this.context = canvas.getContext('2d');\n",
       "\n",
       "    var backingStore =\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        this.context.webkitBackingStorePixelRatio ||\n",
       "        this.context.mozBackingStorePixelRatio ||\n",
       "        this.context.msBackingStorePixelRatio ||\n",
       "        this.context.oBackingStorePixelRatio ||\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        1;\n",
       "\n",
       "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
       "\n",
       "    var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n",
       "        'canvas'\n",
       "    ));\n",
       "    rubberband_canvas.setAttribute(\n",
       "        'style',\n",
       "        'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n",
       "    );\n",
       "\n",
       "    var resizeObserver = new ResizeObserver(function (entries) {\n",
       "        var nentries = entries.length;\n",
       "        for (var i = 0; i < nentries; i++) {\n",
       "            var entry = entries[i];\n",
       "            var width, height;\n",
       "            if (entry.contentBoxSize) {\n",
       "                width = entry.contentBoxSize.inlineSize;\n",
       "                height = entry.contentBoxSize.blockSize;\n",
       "            } else {\n",
       "                width = entry.contentRect.width;\n",
       "                height = entry.contentRect.height;\n",
       "            }\n",
       "\n",
       "            // Keep the size of the canvas and rubber band canvas in sync with\n",
       "            // the canvas container.\n",
       "            canvas.setAttribute('width', width * mpl.ratio);\n",
       "            canvas.setAttribute('height', height * mpl.ratio);\n",
       "            canvas.setAttribute(\n",
       "                'style',\n",
       "                'width: ' + width + 'px; height: ' + height + 'px;'\n",
       "            );\n",
       "\n",
       "            rubberband_canvas.setAttribute('width', width);\n",
       "            rubberband_canvas.setAttribute('height', height);\n",
       "\n",
       "            // And update the size in Python. We ignore the initial 0/0 size\n",
       "            // that occurs as the element is placed into the DOM, which should\n",
       "            // otherwise not happen due to the minimum size styling.\n",
       "            if (width != 0 && height != 0) {\n",
       "                fig.request_resize(width, height);\n",
       "            }\n",
       "        }\n",
       "    });\n",
       "    resizeObserver.observe(canvas_div);\n",
       "\n",
       "    function on_mouse_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.mouse_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousedown',\n",
       "        on_mouse_event_closure('button_press')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseup',\n",
       "        on_mouse_event_closure('button_release')\n",
       "    );\n",
       "    // Throttle sequential mouse events to 1 every 20ms.\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousemove',\n",
       "        on_mouse_event_closure('motion_notify')\n",
       "    );\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseenter',\n",
       "        on_mouse_event_closure('figure_enter')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseleave',\n",
       "        on_mouse_event_closure('figure_leave')\n",
       "    );\n",
       "\n",
       "    canvas_div.addEventListener('wheel', function (event) {\n",
       "        if (event.deltaY < 0) {\n",
       "            event.step = 1;\n",
       "        } else {\n",
       "            event.step = -1;\n",
       "        }\n",
       "        on_mouse_event_closure('scroll')(event);\n",
       "    });\n",
       "\n",
       "    canvas_div.appendChild(canvas);\n",
       "    canvas_div.appendChild(rubberband_canvas);\n",
       "\n",
       "    this.rubberband_context = rubberband_canvas.getContext('2d');\n",
       "    this.rubberband_context.strokeStyle = '#000000';\n",
       "\n",
       "    this._resize_canvas = function (width, height, forward) {\n",
       "        if (forward) {\n",
       "            canvas_div.style.width = width + 'px';\n",
       "            canvas_div.style.height = height + 'px';\n",
       "        }\n",
       "    };\n",
       "\n",
       "    // Disable right mouse context menu.\n",
       "    this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n",
       "        return false;\n",
       "    });\n",
       "\n",
       "    function set_focus() {\n",
       "        canvas.focus();\n",
       "        canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    window.setTimeout(set_focus, 100);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'mpl-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'mpl-button-group';\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'mpl-button-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        var button = (fig.buttons[name] = document.createElement('button'));\n",
       "        button.classList = 'mpl-widget';\n",
       "        button.setAttribute('role', 'button');\n",
       "        button.setAttribute('aria-disabled', 'false');\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "\n",
       "        var icon_img = document.createElement('img');\n",
       "        icon_img.src = '_images/' + image + '.png';\n",
       "        icon_img.srcset = '_images/' + image + '_large.png 2x';\n",
       "        icon_img.alt = tooltip;\n",
       "        button.appendChild(icon_img);\n",
       "\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    var fmt_picker = document.createElement('select');\n",
       "    fmt_picker.classList = 'mpl-widget';\n",
       "    toolbar.appendChild(fmt_picker);\n",
       "    this.format_dropdown = fmt_picker;\n",
       "\n",
       "    for (var ind in mpl.extensions) {\n",
       "        var fmt = mpl.extensions[ind];\n",
       "        var option = document.createElement('option');\n",
       "        option.selected = fmt === mpl.default_extension;\n",
       "        option.innerHTML = fmt;\n",
       "        fmt_picker.appendChild(option);\n",
       "    }\n",
       "\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n",
       "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
       "    // which will in turn request a refresh of the image.\n",
       "    this.send_message('resize', { width: x_pixels, height: y_pixels });\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_message = function (type, properties) {\n",
       "    properties['type'] = type;\n",
       "    properties['figure_id'] = this.id;\n",
       "    this.ws.send(JSON.stringify(properties));\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_draw_message = function () {\n",
       "    if (!this.waiting) {\n",
       "        this.waiting = true;\n",
       "        this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    var format_dropdown = fig.format_dropdown;\n",
       "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
       "    fig.ondownload(fig, format);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_resize = function (fig, msg) {\n",
       "    var size = msg['size'];\n",
       "    if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n",
       "        fig._resize_canvas(size[0], size[1], msg['forward']);\n",
       "        fig.send_message('refresh', {});\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n",
       "    var x0 = msg['x0'] / mpl.ratio;\n",
       "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
       "    var x1 = msg['x1'] / mpl.ratio;\n",
       "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
       "    x0 = Math.floor(x0) + 0.5;\n",
       "    y0 = Math.floor(y0) + 0.5;\n",
       "    x1 = Math.floor(x1) + 0.5;\n",
       "    y1 = Math.floor(y1) + 0.5;\n",
       "    var min_x = Math.min(x0, x1);\n",
       "    var min_y = Math.min(y0, y1);\n",
       "    var width = Math.abs(x1 - x0);\n",
       "    var height = Math.abs(y1 - y0);\n",
       "\n",
       "    fig.rubberband_context.clearRect(\n",
       "        0,\n",
       "        0,\n",
       "        fig.canvas.width / mpl.ratio,\n",
       "        fig.canvas.height / mpl.ratio\n",
       "    );\n",
       "\n",
       "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n",
       "    // Updates the figure title.\n",
       "    fig.header.textContent = msg['label'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n",
       "    var cursor = msg['cursor'];\n",
       "    switch (cursor) {\n",
       "        case 0:\n",
       "            cursor = 'pointer';\n",
       "            break;\n",
       "        case 1:\n",
       "            cursor = 'default';\n",
       "            break;\n",
       "        case 2:\n",
       "            cursor = 'crosshair';\n",
       "            break;\n",
       "        case 3:\n",
       "            cursor = 'move';\n",
       "            break;\n",
       "    }\n",
       "    fig.rubberband_canvas.style.cursor = cursor;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_message = function (fig, msg) {\n",
       "    fig.message.textContent = msg['message'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n",
       "    // Request the server to send over a new figure.\n",
       "    fig.send_draw_message();\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n",
       "    fig.image_mode = msg['mode'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n",
       "    for (var key in msg) {\n",
       "        if (!(key in fig.buttons)) {\n",
       "            continue;\n",
       "        }\n",
       "        fig.buttons[key].disabled = !msg[key];\n",
       "        fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n",
       "    if (msg['mode'] === 'PAN') {\n",
       "        fig.buttons['Pan'].classList.add('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    } else if (msg['mode'] === 'ZOOM') {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.add('active');\n",
       "    } else {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Called whenever the canvas gets updated.\n",
       "    this.send_message('ack', {});\n",
       "};\n",
       "\n",
       "// A function to construct a web socket function for onmessage handling.\n",
       "// Called in the figure constructor.\n",
       "mpl.figure.prototype._make_on_message_function = function (fig) {\n",
       "    return function socket_on_message(evt) {\n",
       "        if (evt.data instanceof Blob) {\n",
       "            /* FIXME: We get \"Resource interpreted as Image but\n",
       "             * transferred with MIME type text/plain:\" errors on\n",
       "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
       "             * to be part of the websocket stream */\n",
       "            evt.data.type = 'image/png';\n",
       "\n",
       "            /* Free the memory for the previous frames */\n",
       "            if (fig.imageObj.src) {\n",
       "                (window.URL || window.webkitURL).revokeObjectURL(\n",
       "                    fig.imageObj.src\n",
       "                );\n",
       "            }\n",
       "\n",
       "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
       "                evt.data\n",
       "            );\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        } else if (\n",
       "            typeof evt.data === 'string' &&\n",
       "            evt.data.slice(0, 21) === 'data:image/png;base64'\n",
       "        ) {\n",
       "            fig.imageObj.src = evt.data;\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        var msg = JSON.parse(evt.data);\n",
       "        var msg_type = msg['type'];\n",
       "\n",
       "        // Call the  \"handle_{type}\" callback, which takes\n",
       "        // the figure and JSON message as its only arguments.\n",
       "        try {\n",
       "            var callback = fig['handle_' + msg_type];\n",
       "        } catch (e) {\n",
       "            console.log(\n",
       "                \"No handler for the '\" + msg_type + \"' message type: \",\n",
       "                msg\n",
       "            );\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        if (callback) {\n",
       "            try {\n",
       "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
       "                callback(fig, msg);\n",
       "            } catch (e) {\n",
       "                console.log(\n",
       "                    \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n",
       "                    e,\n",
       "                    e.stack,\n",
       "                    msg\n",
       "                );\n",
       "            }\n",
       "        }\n",
       "    };\n",
       "};\n",
       "\n",
       "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
       "mpl.findpos = function (e) {\n",
       "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
       "    var targ;\n",
       "    if (!e) {\n",
       "        e = window.event;\n",
       "    }\n",
       "    if (e.target) {\n",
       "        targ = e.target;\n",
       "    } else if (e.srcElement) {\n",
       "        targ = e.srcElement;\n",
       "    }\n",
       "    if (targ.nodeType === 3) {\n",
       "        // defeat Safari bug\n",
       "        targ = targ.parentNode;\n",
       "    }\n",
       "\n",
       "    // pageX,Y are the mouse positions relative to the document\n",
       "    var boundingRect = targ.getBoundingClientRect();\n",
       "    var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n",
       "    var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n",
       "\n",
       "    return { x: x, y: y };\n",
       "};\n",
       "\n",
       "/*\n",
       " * return a copy of an object with only non-object keys\n",
       " * we need this to avoid circular references\n",
       " * http://stackoverflow.com/a/24161582/3208463\n",
       " */\n",
       "function simpleKeys(original) {\n",
       "    return Object.keys(original).reduce(function (obj, key) {\n",
       "        if (typeof original[key] !== 'object') {\n",
       "            obj[key] = original[key];\n",
       "        }\n",
       "        return obj;\n",
       "    }, {});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.mouse_event = function (event, name) {\n",
       "    var canvas_pos = mpl.findpos(event);\n",
       "\n",
       "    if (name === 'button_press') {\n",
       "        this.canvas.focus();\n",
       "        this.canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    var x = canvas_pos.x * mpl.ratio;\n",
       "    var y = canvas_pos.y * mpl.ratio;\n",
       "\n",
       "    this.send_message(name, {\n",
       "        x: x,\n",
       "        y: y,\n",
       "        button: event.button,\n",
       "        step: event.step,\n",
       "        guiEvent: simpleKeys(event),\n",
       "    });\n",
       "\n",
       "    /* This prevents the web browser from automatically changing to\n",
       "     * the text insertion cursor when the button is pressed.  We want\n",
       "     * to control all of the cursor setting manually through the\n",
       "     * 'cursor' event from matplotlib */\n",
       "    event.preventDefault();\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n",
       "    // Handle any extra behaviour associated with a key event\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.key_event = function (event, name) {\n",
       "    // Prevent repeat events\n",
       "    if (name === 'key_press') {\n",
       "        if (event.which === this._key) {\n",
       "            return;\n",
       "        } else {\n",
       "            this._key = event.which;\n",
       "        }\n",
       "    }\n",
       "    if (name === 'key_release') {\n",
       "        this._key = null;\n",
       "    }\n",
       "\n",
       "    var value = '';\n",
       "    if (event.ctrlKey && event.which !== 17) {\n",
       "        value += 'ctrl+';\n",
       "    }\n",
       "    if (event.altKey && event.which !== 18) {\n",
       "        value += 'alt+';\n",
       "    }\n",
       "    if (event.shiftKey && event.which !== 16) {\n",
       "        value += 'shift+';\n",
       "    }\n",
       "\n",
       "    value += 'k';\n",
       "    value += event.which.toString();\n",
       "\n",
       "    this._key_event_extra(event, name);\n",
       "\n",
       "    this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n",
       "    if (name === 'download') {\n",
       "        this.handle_save(this, null);\n",
       "    } else {\n",
       "        this.send_message('toolbar_button', { name: name });\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n",
       "    this.message.textContent = tooltip;\n",
       "};\n",
       "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
       "\n",
       "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
       "\n",
       "mpl.default_extension = \"png\";/* global mpl */\n",
       "\n",
       "var comm_websocket_adapter = function (comm) {\n",
       "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
       "    // object with the appropriate methods. Currently this is a non binary\n",
       "    // socket, so there is still some room for performance tuning.\n",
       "    var ws = {};\n",
       "\n",
       "    ws.close = function () {\n",
       "        comm.close();\n",
       "    };\n",
       "    ws.send = function (m) {\n",
       "        //console.log('sending', m);\n",
       "        comm.send(m);\n",
       "    };\n",
       "    // Register the callback with on_msg.\n",
       "    comm.on_msg(function (msg) {\n",
       "        //console.log('receiving', msg['content']['data'], msg);\n",
       "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
       "        ws.onmessage(msg['content']['data']);\n",
       "    });\n",
       "    return ws;\n",
       "};\n",
       "\n",
       "mpl.mpl_figure_comm = function (comm, msg) {\n",
       "    // This is the function which gets called when the mpl process\n",
       "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
       "\n",
       "    var id = msg.content.data.id;\n",
       "    // Get hold of the div created by the display call when the Comm\n",
       "    // socket was opened in Python.\n",
       "    var element = document.getElementById(id);\n",
       "    var ws_proxy = comm_websocket_adapter(comm);\n",
       "\n",
       "    function ondownload(figure, _format) {\n",
       "        window.open(figure.canvas.toDataURL());\n",
       "    }\n",
       "\n",
       "    var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n",
       "\n",
       "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
       "    // web socket which is closed, not our websocket->open comm proxy.\n",
       "    ws_proxy.onopen();\n",
       "\n",
       "    fig.parent_element = element;\n",
       "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
       "    if (!fig.cell_info) {\n",
       "        console.error('Failed to find cell for figure', id, fig);\n",
       "        return;\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_close = function (fig, msg) {\n",
       "    var width = fig.canvas.width / mpl.ratio;\n",
       "    fig.root.removeEventListener('remove', this._remove_fig_handler);\n",
       "\n",
       "    // Update the output cell to use the data from the current canvas.\n",
       "    fig.push_to_output();\n",
       "    var dataURL = fig.canvas.toDataURL();\n",
       "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
       "    // the notebook keyboard shortcuts fail.\n",
       "    IPython.keyboard_manager.enable();\n",
       "    fig.parent_element.innerHTML =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "    fig.close_ws(fig, msg);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.close_ws = function (fig, msg) {\n",
       "    fig.send_message('closing', msg);\n",
       "    // fig.ws.close()\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n",
       "    // Turn the data on the canvas into data in the output cell.\n",
       "    var width = this.canvas.width / mpl.ratio;\n",
       "    var dataURL = this.canvas.toDataURL();\n",
       "    this.cell_info[1]['text/html'] =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Tell IPython that the notebook contents must change.\n",
       "    IPython.notebook.set_dirty(true);\n",
       "    this.send_message('ack', {});\n",
       "    var fig = this;\n",
       "    // Wait a second, then push the new image to the DOM so\n",
       "    // that it is saved nicely (might be nice to debounce this).\n",
       "    setTimeout(function () {\n",
       "        fig.push_to_output();\n",
       "    }, 1000);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'btn-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'btn-group';\n",
       "    var button;\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'btn-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        button = fig.buttons[name] = document.createElement('button');\n",
       "        button.classList = 'btn btn-default';\n",
       "        button.href = '#';\n",
       "        button.title = name;\n",
       "        button.innerHTML = '<i class=\"fa ' + image + ' fa-lg\"></i>';\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    // Add the status bar.\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message pull-right';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "\n",
       "    // Add the close button to the window.\n",
       "    var buttongrp = document.createElement('div');\n",
       "    buttongrp.classList = 'btn-group inline pull-right';\n",
       "    button = document.createElement('button');\n",
       "    button.classList = 'btn btn-mini btn-primary';\n",
       "    button.href = '#';\n",
       "    button.title = 'Stop Interaction';\n",
       "    button.innerHTML = '<i class=\"fa fa-power-off icon-remove icon-large\"></i>';\n",
       "    button.addEventListener('click', function (_evt) {\n",
       "        fig.handle_close(fig, {});\n",
       "    });\n",
       "    button.addEventListener(\n",
       "        'mouseover',\n",
       "        on_mouseover_closure('Stop Interaction')\n",
       "    );\n",
       "    buttongrp.appendChild(button);\n",
       "    var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n",
       "    titlebar.insertBefore(buttongrp, titlebar.firstChild);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._remove_fig_handler = function () {\n",
       "    this.close_ws(this, {});\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (el) {\n",
       "    el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n",
       "    el.addEventListener('remove', this._remove_fig_handler);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (el) {\n",
       "    // this is important to make the div 'focusable\n",
       "    el.setAttribute('tabindex', 0);\n",
       "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
       "    // off when our div gets focus\n",
       "\n",
       "    // location in version 3\n",
       "    if (IPython.notebook.keyboard_manager) {\n",
       "        IPython.notebook.keyboard_manager.register_events(el);\n",
       "    } else {\n",
       "        // location in version 2\n",
       "        IPython.keyboard_manager.register_events(el);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (event, _name) {\n",
       "    var manager = IPython.notebook.keyboard_manager;\n",
       "    if (!manager) {\n",
       "        manager = IPython.keyboard_manager;\n",
       "    }\n",
       "\n",
       "    // Check for shift+enter\n",
       "    if (event.shiftKey && event.which === 13) {\n",
       "        this.canvas_div.blur();\n",
       "        // select the cell after this one\n",
       "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
       "        IPython.notebook.select(index + 1);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    fig.ondownload(fig, null);\n",
       "};\n",
       "\n",
       "mpl.find_output_cell = function (html_output) {\n",
       "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
       "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
       "    // IPython event is triggered only after the cells have been serialised, which for\n",
       "    // our purposes (turning an active figure into a static one), is too late.\n",
       "    var cells = IPython.notebook.get_cells();\n",
       "    var ncells = cells.length;\n",
       "    for (var i = 0; i < ncells; i++) {\n",
       "        var cell = cells[i];\n",
       "        if (cell.cell_type === 'code') {\n",
       "            for (var j = 0; j < cell.output_area.outputs.length; j++) {\n",
       "                var data = cell.output_area.outputs[j];\n",
       "                if (data.data) {\n",
       "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
       "                    data = data.data;\n",
       "                }\n",
       "                if (data['text/html'] === html_output) {\n",
       "                    return [cell, data, j];\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "};\n",
       "\n",
       "// Register the function which deals with the matplotlib target/channel.\n",
       "// The kernel may be null if the page has been refreshed.\n",
       "if (IPython.notebook.kernel !== null) {\n",
       "    IPython.notebook.kernel.comm_manager.register_target(\n",
       "        'matplotlib',\n",
       "        mpl.mpl_figure_comm\n",
       "    );\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"\" width=\"1500\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from loss import inverse_model, direct_model\n",
    "from data import domain\n",
    "import jax.numpy as jnp\n",
    "import numpy as np\n",
    "\n",
    "inverse_params = agent.params[1]\n",
    "x_test = jnp.linspace(*domain[:, 0], 100).reshape((-1, 1))\n",
    "a_fn = lambda x: 1+np.exp(1/4*x**2)\n",
    "a_pred = inverse_model(inverse_params, x_test)\n",
    "a_true = a_fn(x_test)\n",
    "\n",
    "direct_params = agent.params[0]\n",
    "t_test = domain[1, 1]*jnp.ones_like(x_test)\n",
    "uv_pred = direct_model(direct_params, jnp.hstack([x_test, t_test]))\n",
    "\n",
    "from scipy.io import loadmat\n",
    "data_true = loadmat(\"problem2_4_snapshot_epsilon_1e-12.mat\")\n",
    "u_true, v_true = data_true[\"u_snapshots\"][:, -1], data_true[\"v_snapshots\"][:, -1]\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib notebook\n",
    "\n",
    "f, ax = plt.subplots(1, 3, figsize = (15, 5))\n",
    "ax[0].plot(x_test, a_pred, label = \"pred\")\n",
    "ax[0].plot(x_test, a_true, label = \"true\")\n",
    "ax[0].set_title(\"a\")\n",
    "ax[1].plot(x_test, uv_pred[:, 0:1], label = \"pred\")\n",
    "ax[1].plot(x_test, u_true, label = \"true\")\n",
    "ax[1].set_title(\"u\")\n",
    "ax[2].plot(x_test, uv_pred[:, 1:2], label = \"pred\")\n",
    "ax[2].plot(x_test, v_true, label = \"true\")\n",
    "ax[2].set_title(\"v\")\n",
    "for ax_ in ax:\n",
    "\tax_.legend()\n",
    "\tax_.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
